{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../..')\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "import torch.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.loggers import WandbLogger\n",
    "from pytorch_lightning.metrics.functional import accuracy\n",
    "import torchdiffeq\n",
    "\n",
    "from torchdyn.models import *; from torchdyn import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size=64\n",
    "size=28\n",
    "path_to_data='../../data/mnist_data'\n",
    "\n",
    "all_transforms = transforms.Compose([\n",
    "    transforms.Resize(size),\n",
    "    transforms.ToTensor(),\n",
    "])\n",
    "\n",
    "train_data = datasets.MNIST(path_to_data, train=True, download=True,\n",
    "                            transform=all_transforms)\n",
    "test_data = datasets.MNIST(path_to_data, train=False,\n",
    "                           transform=all_transforms)\n",
    "\n",
    "trainloader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "testloader = DataLoader(test_data, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Learner(pl.LightningModule):\n",
    "    def __init__(self, model:nn.Module):\n",
    "        super().__init__()\n",
    "        self.lr = 1e-3\n",
    "        self.model = model\n",
    "        self.iters = 0\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "    \n",
    "    def training_step(self, batch, batch_idx):\n",
    "        self.iters += 1\n",
    "        x, y = batch   \n",
    "        x, y = x.to(device), y.to(device)\n",
    "        y_hat = self.model(x)   \n",
    "        loss = nn.CrossEntropyLoss()(y_hat, y)\n",
    "        \n",
    "        acc = accuracy(y_hat, y)\n",
    "        nfe = model[2].nfe ; model[2].nfe = 0\n",
    "        tqdm_dict = {'train_loss': loss, 'accuracy': acc, 'NFE': nfe}\n",
    "        \n",
    "        self.logger.experiment.log(step=self.iters, row=tqdm_dict)\n",
    "        return {'loss': loss}   \n",
    "\n",
    "    def test_step(self, batch, batch_nb):\n",
    "        x, y = batch\n",
    "        x, y = x.to(device), y.to(device)\n",
    "        y_hat = self(x)\n",
    "        test_acc = accuracy(y_hat, y)\n",
    "        test_loss = nn.CrossEntropyLoss()(y_hat, y)\n",
    "        metrics = {'test_loss': test_loss, 'test_accuracy': test_acc}\n",
    "        self.logger.experiment.log(step=self.iters, row=metrics)\n",
    "        return {'test_loss': test_loss}\n",
    "\n",
    "    def test_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()\n",
    "        avg_acc = torch.stack([x['test_accuracy'] for x in outputs]).mean()\n",
    "        metrics = {'avg_test_loss': avg_loss, 'avg_test_accuracy': avg_acc}\n",
    "        self.logger.experiment.log(step=self.iters, row=metrics)\n",
    "        return {'avg_test_loss': avg_loss}\n",
    "    \n",
    "    def validation_step(self, batch, batch_nb):\n",
    "        x, y = batch\n",
    "        x, y = x.to(device), y.to(device)\n",
    "        y_hat = self(x)\n",
    "        val_acc = accuracy(y_hat, y)\n",
    "        val_loss = nn.CrossEntropyLoss()(y_hat, y)\n",
    "        metrics = {'val_loss': val_loss, 'val_accuracy': val_acc}\n",
    "        self.logger.experiment.log(step=self.iters, row=metrics)\n",
    "        return {'val_loss': val_loss, 'val_accuracy': val_acc}\n",
    "\n",
    "    def validation_epoch_end(self, outputs):\n",
    "        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n",
    "        avg_acc = torch.stack([x['val_accuracy'] for x in outputs]).mean()\n",
    "        metrics = {'avg_val_loss': avg_loss, 'avg_val_accuracy': avg_acc}\n",
    "        self.logger.experiment.log(step=self.iters, row=metrics)\n",
    "        return {'avg_val_loss': avg_loss}\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        return torch.optim.Adam(self.parameters(), lr=self.lr, weight_decay=1e-4)\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        self.trloader_len = len(trainloader)\n",
    "        return trainloader\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        self.vloader_len = len(testloader) # using test as val here\n",
    "        return testloader\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        self.test_tloader_len = len(testloader)\n",
    "        return testloader"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "func = nn.Sequential(DataControl(),\n",
    "                     nn.Conv2d(12, 6, 3, padding=1),\n",
    "                     nn.Tanh(),\n",
    "                     ).to(device)\n",
    "\n",
    "neuralDE = NeuralDE(func, \n",
    "                   solver='rk4',\n",
    "                   sensitivity='adjoint',\n",
    "                   s_span=torch.linspace(0, 1, 30)).to(device)\n",
    "\n",
    "model = nn.Sequential(Augmenter(augment_dims=5),\n",
    "                      nn.BatchNorm2d(6),\n",
    "                      neuralDE,\n",
    "                      nn.Conv2d(6, 1, 3, padding=1),\n",
    "                      nn.Flatten(),\n",
    "                      nn.Dropout(p=0.5),\n",
    "                      nn.Linear(28*28, 10)).to(device)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger = WandbLogger(project='torchdyn-mnist-bench', log_model=True) # feel free to comment out or use a different logging scheme :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
       "                Project page: <a href=\"https://app.wandb.ai/zymrael/torchdyn-mnist-bench\" target=\"_blank\">https://app.wandb.ai/zymrael/torchdyn-mnist-bench</a><br/>\n",
       "                Run page: <a href=\"https://app.wandb.ai/zymrael/torchdyn-mnist-bench/runs/3g406dpw\" target=\"_blank\">https://app.wandb.ai/zymrael/torchdyn-mnist-bench/runs/3g406dpw</a><br/>\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  | Name  | Type       | Params\n",
      "-------------------------------------\n",
      "0 | model | Sequential | 8 K   \n",
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: Your val_dataloader has shuffle=True, it is best practice to turn this off for validation and test dataloaders.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, val dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 20 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validation sanity check', layout=Layout…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, train dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 20 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31fcdcebc8a74b7d881e510b567a7e2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Training', layout=Layout(flex='2'), max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: RuntimeWarning: The metric you returned None must be a `torch.Tensor` instance, checkpoint not saved HINT: what is the value of val_loss in validation_epoch_end()?\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: RuntimeWarning: Can save best model only with val_loss available, skipping.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Validating', layout=Layout(flex='2'), m…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: Detected KeyboardInterrupt, attempting graceful shutdown...\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learn = Learner(model)\n",
    "trainer = pl.Trainer(max_epochs=20,\n",
    "                     logger=logger,\n",
    "                     benchmark=True,\n",
    "                     #limit_test_batches=0.25,\n",
    "                     val_check_interval=0.05,\n",
    "                     gpus=1,  \n",
    "                     progress_bar_refresh_rate=1\n",
    "                     )\n",
    "\n",
    "trainer.fit(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: Your test_dataloader has shuffle=True, it is best practice to turn this off for validation and test dataloaders.\n",
      "  warnings.warn(*args, **kwargs)\n",
      "/home/jyp/michael_dev/testenv/lib/python3.7/site-packages/pytorch_lightning/utilities/distributed.py:25: UserWarning: The dataloader, test dataloader 0, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 20 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  warnings.warn(*args, **kwargs)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "554acfe2869f4dbc87aaca27a845a316",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Testing', layout=Layout(flex='2'), max=…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0424, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0958, device='cuda:0'), 'test_accuracy': tensor(0.9219, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0706, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0696, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0306, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1104, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0338, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0368, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0626, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0249, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0182, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0207, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0062, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0241, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0837, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1190, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1998, device='cuda:0'), 'test_accuracy': tensor(0.8906, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0495, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0449, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0150, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0650, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0414, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0453, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1097, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1140, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1946, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1308, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0598, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0131, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0303, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0429, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0682, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0832, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0425, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0374, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0089, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0372, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0764, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1516, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0827, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0914, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0210, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1059, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0141, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0627, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0214, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0115, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0243, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0731, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0836, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0723, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0222, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0205, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0980, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0335, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1184, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1263, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0342, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1412, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0374, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0096, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0599, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0261, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0451, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0446, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0550, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0553, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0668, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0737, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0605, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0338, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0616, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0270, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1025, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0078, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0128, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0553, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1706, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0282, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0590, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0719, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0456, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0233, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0316, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0251, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1414, device='cuda:0'), 'test_accuracy': tensor(0.9219, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0127, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0869, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.2641, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1434, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0446, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0854, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1045, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0346, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0952, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0591, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0178, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0885, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0168, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0116, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0122, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0132, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1134, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1411, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0147, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0719, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0239, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0477, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0849, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0210, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0513, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0440, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0505, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0190, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0627, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0475, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0377, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1319, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1034, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0256, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0911, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0336, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0150, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0193, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0331, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0800, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0332, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0631, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0236, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0149, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0323, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0895, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0069, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0156, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0134, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0157, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1282, device='cuda:0'), 'test_accuracy': tensor(0.9531, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0295, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0924, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0070, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0336, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0942, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0594, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1008, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0224, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0474, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0201, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0272, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0190, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0337, device='cuda:0'), 'test_accuracy': tensor(0.9844, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0742, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0116, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1031, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0217, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.1281, device='cuda:0'), 'test_accuracy': tensor(0.9375, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0751, device='cuda:0'), 'test_accuracy': tensor(0.9688, device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'test_loss': tensor(0.0213, device='cuda:0'), 'test_accuracy': tensor(1., device='cuda:0')}.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Adding to old History rows isn't currently supported.  Step 18760 < 18761; dropping {'avg_test_loss': tensor(0.0580, device='cuda:0'), 'avg_test_accuracy': tensor(0.9802, device='cuda:0')}.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "TEST RESULTS\n",
      "{'avg_test_accuracy': tensor(0.9802, device='cuda:0'),\n",
      " 'avg_test_loss': tensor(0.0580, device='cuda:0')}\n",
      "--------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trainer.test(learn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py37",
   "language": "python",
   "name": "py37"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
